package org.opendream.roti.lingpipe;

import com.aliasi.tokenizer.Tokenizer;

public class CharTokenizer extends Tokenizer {

	private final char[] mChars;
	private final int mLastPosition;
	private int mPosition;
	private int lastPosition = -1;

	public CharTokenizer(char[] ch, int offset, int length) {
		mChars = ch;
		mPosition = offset;
		mLastPosition = offset + length;
	}
	
    public int lastTokenStartPosition() {
        return lastPosition;
    }

	public String nextWhitespace() {
		StringBuffer sb = new StringBuffer();
		while (hasMoreCharacters() && Character.isWhitespace(currentChar())) {
			sb.append(currentChar());
			++mPosition;
		}
		return sb.toString();
	}

	public String nextToken() {
		skipWhitespace();
		if (!hasMoreCharacters())
			return null;
		lastPosition = mPosition;
		return new String(new char[] { mChars[mPosition++] });
	}

	private void skipWhitespace() {
		while (hasMoreCharacters() && Character.isWhitespace(currentChar()))
			++mPosition;
	}

	private boolean hasMoreCharacters() {
		return mPosition < mLastPosition;
	}

	private char currentChar() {
		return mChars[mPosition];
	}

}